package com.example.webmagic;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;

public class Test implements PageProcessor {
    private static final Logger logger = LoggerFactory.getLogger(Test.class);


    // 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000);

    @Override
    public void process(Page page) {
        /* page.putField("title", page.getHtml().css("a[rel]", "text").all());
         *//*System.out.println(page.getHtml().css("a[rel]"));*/
        String url = page.getUrl().get();
        if (url.contains(".html")) {
            this.getContent(page);
        } else if (url.equals("https://www.147xs.org/")) {
            List<String> all = page.getHtml().links().regex(".*book/\\d+/").all();
            page.addTargetRequests(all);
            page.setSkip(true);
        } else {
            List<String> list = page.getHtml().links().regex(".*\\d+\\.html").all();
            page.addTargetRequests(list);
            page.setSkip(true);

        }
    }

    public void getContent(Page page) {
        String bookname = page.getHtml().xpath("//div[@class='con_top']/a[3]/text()").get();
        String title = page.getHtml()
                .xpath("//div[@class='bookname']/h1/text()").get();
        //内容
        List<String> all = page.getHtml()
                .xpath("//div[@id='content']//p/text()").all();
        //通过键值对的方式存入，用于后面写入文件时用到这些属性
        page.putField("bookname", bookname);
        page.putField("title", title.replaceAll("[/?\\\\]", ""));
        page.putField("content", all);
    }

    @Override
    public Site getSite() {
        return site;
    }

    static class SaveCustom implements Pipeline {

        @Override
        public void process(ResultItems resultItems, Task task) {
            //通过key获取值
            String bookname = resultItems.get("bookname");
            String title = resultItems.get("title");
            List<String> contents = resultItems.get("content");

            File file = new File("D:\\web\\" + bookname);
            if (!file.exists()) {
                file.mkdir();
            }

            try {
                FileOutputStream fileOutputStream = new FileOutputStream(file.getAbsolutePath() + "\\" + title + ".txt");
                for (String content : contents) {
                    String replace = content + "\n\n";
                    fileOutputStream.write(replace.getBytes(StandardCharsets.UTF_8));

                }
                fileOutputStream.flush();
                fileOutputStream.close();
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }

    public static void main(String[] args) {
        Spider.create(new Test()).addUrl("https://www.147xs.org/book/132845/")
                .addPipeline(new ConsolePipeline())
                .addPipeline(new SaveCustom())
                .thread(50)
                .run();
    }
}
